{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint-1000  checkpoint-3000  checkpoint-500   checkpoint-6500\r\n",
      "checkpoint-1500  checkpoint-3500  checkpoint-5000  checkpoint-7000\r\n",
      "checkpoint-2000  checkpoint-4000  checkpoint-5500  final_merged_checkpoint\r\n",
      "checkpoint-2500  checkpoint-4500  checkpoint-6000  runs\r\n"
     ]
    }
   ],
   "source": [
    "!ls results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-30 13:57:56.487182: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-07-30 13:57:56.663625: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-07-30 13:57:57.468645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-07-30 13:57:57.468691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-07-30 13:57:57.468694: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "from peft import AutoPeftModelForCausalLM\n",
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc56ec22ba4d43c282c42a76b090071e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    './results/checkpoint-7000', device_map=\"auto\", torch_dtype=torch.bfloat16)\n",
    "model = model.merge_and_unload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-2-7b-hf', trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "template = {\n",
    "    'description': 'Template used by Malaya.',\n",
    "    'prompt_input': 'Di bawah ialah arahan yang menerangkan tugasan, termasuk dengan input yang menyediakan konteks lanjut. Tulis jawapan yang sesuai dengan arahan tersebut.\\n\\n### Arahan:\\n{instruction}\\n\\n### Input:\\n{input}\\n\\n### Jawapan:\\n',\n",
    "    'prompt_no_input': 'Di bawah ialah arahan yang menerangkan tugasan. Tulis jawapan yang sesuai dengan arahan tersebut.\\n\\n### Arahan:\\n{instruction}\\n\\n### Jawapan:\\n',\n",
    "    'response_split': '### Jawapan:',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[    1,  4671,   289,  1450,   801,   474,   284,   801, 24418,  5403,\n",
       "           343,   574,   286,   759,   574, 11052,   260,   688,   294,   273,\n",
       "         29889, 27415,   275,   432,  1450, 21419,   343,   574,  3999, 29884,\n",
       "          1794,   972,  6249, 24418,  5403,  1935,   344,  4187, 29889,    13,\n",
       "            13,  2277, 29937, 25953,  5403, 29901,    13,    13, 26956,   421,\n",
       "         29968, 29965,  1964, 29909,   323,  1001,  1430, 26788,  2190, 29965,\n",
       "           448, 12693,  9919,   273,  1258,   557,  2766,   413,   300,   392,\n",
       "           375,   273,   338, 29884,   409,  2790,  3249,   286,   996,  1249,\n",
       "         11052,   282,   331,  6574,   262,  2431,   638, 23402, 22318,  1848,\n",
       "           313, 15695, 29897,   972,  6249,   413, 15274,   532,   273,  7655,\n",
       "          6948,   332,   639,  1335,  7889, 29889,    13,    13, 29968,   300,\n",
       "          3357,   349,   331,  6191, 10043,  5061,   996,  6249, 29884, 29892,\n",
       "         12929, 29881,  3536,   348,  3423, 29874,   289,  5968,   532, 29892,\n",
       "           338, 29884,  1935,   344,  4187, 13547,   801,   337,  1388,  6025,\n",
       "           413,  1590,  2606,   652, 29890,   574,  7354, 11052, 13159,  9010,\n",
       "          1153,  3459,   271,  2626,   392,   574,   724,   549,  1935, 21312,\n",
       "           481, 17756,  7655,  1717, 29874,   273, 29889,    13,    13, 29908,\n",
       "         29902, 29874,   298,  1384,   284,   801,   260,   513,   557,   273,\n",
       "          2832,   638,  6025,  1935,  2783,   557,  2930, 29882,   557, 13023,\n",
       "          9919,   273,   343,   574,   413,   300,   392,   375,   273,   338,\n",
       "         29884,   443, 29873,  2679,   286,   996,  1249, 11052,   282,  1590,\n",
       "           574, 29895,   574,   972,  6249,   413,   267,   284,   801,   273,\n",
       "         29889,    13,    13, 29908, 29902,  2146,   297, 29875,  5053,   801,\n",
       "           301,  3304,  6025,  5053,   801,   337,  1388,  6869,   348,   409,\n",
       "           546,  2034, 15187,  1335,   297, 29875,   313,  3946,  9919,   273,\n",
       "         29897,  5516,  4861,   972,  6249, 19119,  4812,   525, 29886,  1590,\n",
       "           574, 29895,   574, 29915, 13023,  1648,   286,   996, 11895,   271,\n",
       "         11052,  3999, 29884,  8088,   338, 29884, 13159,  9010,  1153,  3459,\n",
       "           271,   282,   392,   574,   724,   549,   413,  1022,  1114,   349,\n",
       "         29940,  1699, 29466, 20912,   413,   300,  4106,   652, 29882,   431,\n",
       "           686, 29875,  8882,   279,  3536,   713,   282,  1114, 29489,   262,\n",
       "         17580,    13,    13, 21111,   599, 16212,   297,   278, 14880, 29892,\n",
       "           289, 17698,  2959,   314,  4663,    13,    13,    13,  2277, 29937,\n",
       "           435,  1450, 21419, 29901,    13]], device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"\"\"\n",
    "paragraph `KUALA TERENGGANU - Kerajaan didakwa ketandusan isu sehingga mengaitkan pemimpin Perikatan Nasional (PN) dengan kenyataan berunsur perkauman.\n",
    "\n",
    "Ketua Pemuda Pas Terengganu, Mohd Harun Esa berkata, isu tersebut telah reda dan kembali dibangkitkan supaya rakyat memandang serong terhadap parti berkenaan.\n",
    "\n",
    "\"Ia hanyalah tindakan politik dan terdesak pihak kerajaan yang ketandusan isu untuk mengaitkan pembangkang dengan kesalahan.\n",
    "\n",
    "\"Isu ini sudah lama dan sudah reda namun seperti mereka ini (kerajaan) masih dengan mentaliti 'pembangkang' kerana menghangatkan sesuatu isu supaya rakyat pandang serong kepada PN,\" katanya ketika dihubungi Sinar Harian pada Isnin.`\n",
    "\n",
    "extract all entities in the paragraph, bagi dalam JSON\n",
    "\"\"\"\n",
    "prompt = template['prompt_no_input'].format(instruction=query)\n",
    "inputs = tokenizer(prompt, return_tensors='pt')\n",
    "input_ids = inputs['input_ids'].to(model.device)\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_new_tokens = 300\n",
    "temperature = 0.9\n",
    "top_p = 0.95\n",
    "top_k = 50\n",
    "num_beams = 1\n",
    "do_sample = True\n",
    "with torch.no_grad():\n",
    "    generation_output = model.generate(\n",
    "        input_ids=input_ids,\n",
    "        return_dict_in_generate=True,\n",
    "        output_scores=True,\n",
    "        max_new_tokens=max_new_tokens,\n",
    "        temperature=temperature,\n",
    "        top_p=top_p,\n",
    "        top_k=top_k,\n",
    "        num_beams=num_beams,\n",
    "        do_sample=do_sample,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s> Di bawah ialah arahan yang menerangkan tugasan. Tulis jawapan yang sesuai dengan arahan tersebut.\\n\\n### Arahan:\\n\\nparagraph `KUALA TERENGGANU - Kerajaan didakwa ketandusan isu sehingga mengaitkan pemimpin Perikatan Nasional (PN) dengan kenyataan berunsur perkauman.\\n\\nKetua Pemuda Pas Terengganu, Mohd Harun Esa berkata, isu tersebut telah reda dan kembali dibangkitkan supaya rakyat memandang serong terhadap parti berkenaan.\\n\\n\"Ia hanyalah tindakan politik dan terdesak pihak kerajaan yang ketandusan isu untuk mengaitkan pembangkang dengan kesalahan.\\n\\n\"Isu ini sudah lama dan sudah reda namun seperti mereka ini (kerajaan) masih dengan mentaliti \\'pembangkang\\' kerana menghangatkan sesuatu isu supaya rakyat pandang serong kepada PN,\" katanya ketika dihubungi Sinar Harian pada Isnin.`\\n\\nextract all entities in the paragraph, bagi dalam JSON\\n\\n\\n### Jawapan:\\n[{\"entities\":[{\"entity\":0,\"text\":\"Ketua Pemuda Pas Terengganu, Mohd Harun Esa\",\"offset\":66,\"length\":15,\"entityType\":0},{\"entity\":1,\"text\":\"berkata\",\"offset\":81,\"length\":10,\"entityType\":1},{\"entity\":2,\"text\":\"isu tersebut telah reda dan kembali dibangkitkan supaya rakyat memandang serong terhadap parti berkenaan.\",\"offset\":92,\"length\":39,\"entityType\":2},{\"entity\":3,\"text\":\"Isu ini sudah lama dan sudah reda namun seperti mereka ini (kerajaan) masih dengan mentaliti \\'pembangkang\\' kerana menghangatkan sesuatu isu supaya rakyat pandang serong kepada PN\",\"offset\":131,\"length\":67,\"entityType\":3}]}\\n\\n### Jawapan:\\nEntiti dalam ayat:\\n1. Ketua Pemuda Pas Terengganu, Mohd Harun Esa.\\n2. Berkata.\\n3. Isu tersebut telah reda dan kembali dibangkitkan supaya rakyat memandang ser'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(generation_output.sequences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_merged_dir = './results/final_merged_checkpoint'\n",
    "model.save_pretrained(output_merged_dir, safe_serialization=True, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "config.json\t\t\t  model-00002-of-00002.safetensors\r\n",
      "generation_config.json\t\t  model.safetensors.index.json\r\n",
      "model-00001-of-00002.safetensors\r\n"
     ]
    }
   ],
   "source": [
    "!ls {output_merged_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.8/site-packages/transformers/utils/hub.py:665: UserWarning: The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`).\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68ec390edeb34021a528dd7326f9df1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00001-of-00002.bin:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38191525da7846b5b378f019239db354",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00002-of-00002.bin:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d5b6b3abedc49e6a86bc7dbc1759d34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 2 LFS files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/llama-7b-hf-512-ms-qlora/commit/69407ccbefa0f4aeeb50920033229954ebb974a0', commit_message='Upload LlamaForCausalLM', commit_description='', oid='69407ccbefa0f4aeeb50920033229954ebb974a0', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('llama-7b-hf-512-ms-qlora', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfbf3cf0196f4568a315e853db9d6023",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/llama-7b-hf-512-ms-qlora/commit/4c44042b3046275d86f804216cecb9eee352fa64', commit_message='Upload tokenizer', commit_description='', oid='4c44042b3046275d86f804216cecb9eee352fa64', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('llama-7b-hf-512-ms-qlora', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
